package com.wanghang89.gmail.http.crawler;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.log4j.Logger;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.wanghang89.gmail.bean.AdministraRegion;
import com.wanghang89.gmail.http.Downloader;
import com.wanghang89.gmail.http.DownloaderConstants;
import com.wanghang89.gmail.http.EncodingConstants;
import com.wanghang89.gmail.util.BDCSContext;
import com.wanghang89.gmail.util.db.AdministraRegionDbUtils;
import com.wanghang89.gmail.util.db.DBException;

/**
 * 抓取行政区域划分信息
 * @ClassName AdministraRegionCrawler
 * @Description TODO
 * @author 王航 
 * @date 2013-7-22
 */
public class AdministraRegionCrawler implements DownloaderConstants , EncodingConstants {
	
	private final static String PREFIX = "http://www.xzqh.org/html/";
	private final static String ENTRANCE_URL = "http://www.xzqh.org/html/list/10100.html";
	
	private final static Pattern PATTERN = Pattern.compile("[0-9a-zA-Z，。：！《》]+");
	
	private static AdministraRegionDbUtils arDbUtils = (AdministraRegionDbUtils)BDCSContext.getBean("arDbUtils");
	
	private final static Logger LOGGER =  Logger.getLogger(AdministraRegionCrawler.class);
	
	public static void crawl() throws DBException
	{
		Document doc = Downloader.downloadByGet(ENTRANCE_URL,GBK);
		String pro_select = "div[id=xwzxbox]:eq(0) ul[class=List-2]:eq(0) li:gt(1)";
		Elements pro_elements = doc.select(pro_select);
		Iterator<Element> pro_iterator = pro_elements.iterator();
		long start = System.currentTimeMillis();
		while (pro_iterator.hasNext()) {
			Element pro_element = pro_iterator.next();
			String href = PREFIX+pro_element.select("a").first().attr("href");
			String regionName = pro_element.text();
			Document con_doc = Downloader.downloadByGet(href, GBK);
			Elements con_eles = con_doc.select("div[class=content]");
			if (con_eles!=null&&!con_eles.isEmpty()) {
				Element con_ele = con_eles.first();
				String html = con_ele.html();
				html = html.replaceAll("\n?<!--.*?-->", "").replaceAll("&[a-zA-Z]+;","");
				String[] lines = html.split("\n");
				AdministraRegion ar = new AdministraRegion();
				ar.setName(regionName);
				ar.setRemark(lines[0]);
				ar.setGardeId(1);
				Integer id = arDbUtils.add(ar);
				parseChildren(lines,id);
			}else {
				AdministraRegion ar = new AdministraRegion();
				ar.setName(regionName);
				ar.setGardeId(1);
				arDbUtils.add(ar);
			}
		}
		long end = System.currentTimeMillis();
		LOGGER.debug((end-start)/1000+"s");
	};
	
	private static void parseChildren(String[] lines , Integer superiorId) throws DBException
	{
		Integer spId = superiorId;
		Integer pId = superiorId;
		List<AdministraRegion> ars = new ArrayList<AdministraRegion>();
		int length = lines.length;		
		for (int i = 1; i < length ; i++) {
			String line = lines[i].trim();
			if (line.startsWith("<strong>")) {
				arDbUtils.add(ars);
				ars.clear();
				line = line.replaceAll("<strong>","").replaceAll("</strong>","|");
				String[] line2sa = line.split("\\|");
				if (line2sa.length>0) {
					String regionName = line2sa[0];
					int rni = regionName.indexOf("<br />");
					if (rni!=-1) {
						String region2Name = regionName.substring(0,rni);
						if (isRegionName(region2Name)) {
							AdministraRegion ar = new AdministraRegion();
							ar.setName(region2Name);
							ar.setSuperiorId(spId);
							ar.setGardeId(2);
							arDbUtils.add(ar);		
						}
					}
					if (regionName.indexOf(" ")!=-1) {
						String[] rnsa = regionName.split(" ");
						int rnsal = rnsa.length;
						regionName = rnsa[rnsal-1];
					}
					if (isRegionName(regionName)) {
						AdministraRegion ar = new AdministraRegion();
						ar.setName(regionName);
						ar.setGardeId(2);
						ar.setSuperiorId(spId);						
						if (line2sa.length>1) {
							ar.setRemark(line2sa[1]);
						}
						pId = arDbUtils.add(ar);
					}else {
						pId = spId;
					}
				}
			}else if (line.startsWith("<br")) {
				line = line.replaceAll("[　\\s]+","").replaceAll("[0-9]{10,}","_")
							.replaceAll("[\\（\\(辖]","_").replaceAll("[\\）\\)]","\\$").replace("<br/>","");
				line = line.trim();
				if (line.isEmpty()) {
					continue;
				}
				String[] lsa = line.split("\\$");
				int lsal = lsa.length;
				for (int j = 0; j < lsal; j++) {
					String tmp = lsa[j];
					String[] tsa = tmp.split("_");
					int tsal = tsa.length;
					if (tsal>0) {
						if (!isRegionName(tsa[0])) {
							continue;
						}
						AdministraRegion ar = new AdministraRegion();
						ar.setName(tsa[0]);
						ar.setGardeId(3);
						ar.setSuperiorId(pId);
						if (tsal>1) {
							ar.setRemark(tsa[1]);
						}
						ars.add(ar);
					}
				}
			}
		}
		if (!ars.isEmpty()) {
			arDbUtils.add(ars);
		}
		ars.clear();
		ars = null;
	}
	
	private static boolean isRegionName(String str)
	{
		str = str.replaceAll("(?is)<.*?>","");
		Matcher matcher = PATTERN.matcher(str);
		if (matcher.find()) {
			LOGGER.debug("throw away useless information:"+str);
			return false;
		}
		if (str.indexOf("一览")!=-1) {
			LOGGER.debug("throw away useless information:"+str);
			return false;
		}else if(str.indexOf("直辖")!=-1) {
			LOGGER.debug("throw away useless information:"+str);
			return false;
		}
		return true;
	}
	
	public static void main(String[] args) throws DBException {
		crawl();
	}
}
